Twitter Media Lab

Load in Packages

reading twitter ids

tweet_ids <- dir(here("data", "tw"))
here::here("data", "tw")

[1] "C:/Users/Fabio/Documents/git_proj/altright/data/tw"

tweet_ids_list <- list()
for (jj in seq_along(tweet_ids)) {
  tweet_ids_list[[jj]] <- jsonlite::fromJSON(here("data", "tw", tweet_ids[jj]))
}
enframed_tweets <- enframe(tweet_ids_list)
handlers <- c("ABC", "RealAlexJones", "AnnCoulter", "MsBlaireWhite", "navyhato", "BreitbartNews", "Cernovich", "CNN", "FoxNews", "infowars", "Lauren_Southern", "MSNBC", "PrisonPlanet", "TheRebelTV", "RoamingMil","TRobinsonNewEra", "washingtonpost")
ff <- list()
for (jj in seq_along(handlers)) {
  ff[[jj]] <- cbind(enframed_tweets$value[[jj]], handlers[jj])
}
rlist::list.rbind(ff) %>% 
   as.data.frame()

# REPLACE WITH YOUR API KEY
myapikey <- "Rxly3JKWfdM5osWQenXrxzSQX"
# REPLACE WITH YOUR API SECRET
myapisecret <- "u1NPPaUKdMW11dqwhWVwMQIYwmd42YL8fHTM7YhOHiex7YIku5"
# REPLACE WITH YOUR ACCESS TOKEN
myaccesstoken <- "513351911-HnGVTozLlxCQJh4u0kntUajynJDVBHQP1qNFe588"
# REPLACE WITH YOUR ACCESS TOKEN SECRET
myaccesstokensecret <- "mfNN2NVnd9d0WNmS4tZCz85r6swsCZp2H23JdgHMJdXh1"

Load in Data

myTwitterData <- SocialMediaLab::Authenticate("twitter", 
                              apiKey=consumer_key,
                              apiSecret=consumer_secret, 
                              accessToken=access_token,
                              accessTokenSecret=access_secret)# %>%
  Collect(searchTerm="#prayforsouthafrica| #stopwhitegenocide |#whitegenocide| #17states| #unitetheright| #novaeuropa| #tokyo| #buildthewall| #trumpmemes| #trump| #rightwing| #maga| #makeamericagreatagain| #politics| #patriot| #guns",
          numTweets=20000,
          writeToFile=FALSE,
          verbose=TRUE)
Collect(ego = TRUE,
username = myUsernames)

myTwitterData$text <- iconv(myTwitterData$text, to = 'utf-8')

Filter Data

g_twitter_actor <- myTwitterData %>% Create("actor", writeToFile = TRUE)

g_twitter_actor

Recoding

myTwitterData <- Authenticate("twitter", 
                              apiKey=myapikey,
                              apiSecret=myapisecret, 
                              accessToken=myaccesstoken,
                              accessTokenSecret=myaccesstokensecret) %>%
  Collect(ego = TRUE,
username = c("RichardBSpencer", "realDonaldTrump", "mikeenochsback", "MillennialWoes", "thealthype", "JFGariepy"))

Analysis

#install.packages("rtweet")
library(rtweet)


# REPLACE WITH YOUR API KEY
myapikey <- "Rxly3JKWfdM5osWQenXrxzSQX"
# REPLACE WITH YOUR API SECRET
myapisecret <- "u1NPPaUKdMW11dqwhWVwMQIYwmd42YL8fHTM7YhOHiex7YIku5"
# REPLACE WITH YOUR ACCESS TOKEN
myaccesstoken <- "513351911-HnGVTozLlxCQJh4u0kntUajynJDVBHQP1qNFe588"
# REPLACE WITH YOUR ACCESS TOKEN SECRET
myaccesstokensecret <- "mfNN2NVnd9d0WNmS4tZCz85r6swsCZp2H23JdgHMJdXh1"
# Change the next four lines based on your own consumer_key, consume_secret, access_token, and access_secret. 

access_token <- "   513351911-HnGVTozLlxCQJh4u0kntUajynJDVBHQP1qNFe588"
access_secret <- "  mfNN2NVnd9d0WNmS4tZCz85r6swsCZp2H23JdgHMJdXh1"




tw = twitteR::searchTwitter("xD", n = 1e4, since = '2018-01-01', retryOnRateLimit = 1e3)
d = twitteR::twListToDF(tw)


tmls <- rtweet::get_timelines("196168350", n = 20)

rt <- search_tweets(
  "#rstats", n = 18000, include_rts = FALSE
)

flw <- get_followers("196168350", n = 10)

flw_tml <- vector("list", length(flw$user_id))
for (i in seq_along(flw$user_id)) {
  flw_tml[[i]] <- get_timeline(
    flw$user_id[i], n = 10
  )
  if (i %% 56 == 0L) {
    rl <- rate_limit("get_timeline")
    Sys.sleep(as.numeric(rl$reset, "secs"))
  }
  cat(i, " ")
}

at_rdt <- rtweet::search_tweets(
  "to:Cernovich", 
  n = 5e6,
  since = "2017-01-01",
  until = "2018-01-01",
   retryonratelimit = TRUE

tweets <- userTimeline('MillennialWoes', n = 3200)

# convert status object to data frame for easier manipulation
tweetsdf <- twListToDF(tweets)

tweetsdf %>% 
  mutate(date = lubridate::as_date(created)) %>%
  filter(date > lubridate::as_date("2017-01-01") & date < lubridate::as_date("2018-01-01"))

library(httr)

# 1. Find OAuth settings for twitter:
#    https://dev.twitter.com/docs/auth/oauth
oauth_endpoints("twitter")

# 2. Register an application at https://apps.twitter.com/
#    Make sure to set callback url to "http://127.0.0.1:1410/"
#
#    Replace key and secret below
myapp <- oauth_app("twitter",
  key = "TYrWFPkFAkn4G5BbkWINYw",
  secret = "qjOkmKYU9kWfUFWmekJuu5tztE9aEfLbt26WlhZL8"
)

# 3. Get OAuth credentials
twitter_token <- oauth1.0_token(oauth_endpoints("twitter"), myapp)

## 4. Use API
#req <- GET("https://api.twitter.com/1.1/statuses/home_timeline.json",
#  config(token = twitter_token))
#stop_for_status(req)
#content(req)

library(rtweet)
library(igraph)
library(hrbrthemes)
library(ggraph)
library(tidyverse)
rstats <- search_tweets("#antiwhite", n=20000)

# same as previous recipe
filter(rstats, retweet_count > 0) %>% 
  select(screen_name, mentions_screen_name) %>%
  unnest(mentions_screen_name) %>% 
  filter(!is.na(mentions_screen_name)) %>% 
  graph_from_data_frame() -> rt_g


V(rt_g)$node_label <- unname(ifelse(degree(rt_g)[V(rt_g)] > 20, names(V(rt_g)), "")) 
V(rt_g)$node_size <- unname(ifelse(degree(rt_g)[V(rt_g)] > 20, degree(rt_g), 0)) 

ggraph(rt_g, layout = 'linear', circular = TRUE) + 
  geom_edge_arc(edge_width=0.125, aes(alpha=..index..)) +
  geom_node_label(aes(label=node_label, size=node_size),
                  label.size=0, fill="#ffffff66", segment.colour="springgreen",
                  color="slateblue", repel=TRUE, family=font_rc, fontface="bold") +
  coord_fixed() +
  scale_size_area(trans="sqrt") +
  labs(title="Retweet Relationships", subtitle="Most retweeted screen names labeled. Darkers edges == more retweets. Node size == larger degree") +
  theme_graph(base_family=font_rc) +
  theme(legend.position="none")


ggsave(file = here("images","network5.png"))

map_chr2 <- function(x, .f, ...) {
  map(x, .f, ...) %>% map_if(is_empty, ~ NA_character_) %>% flatten_chr()
}
ellipsize <- function(x, n = 20) {
  ifelse(str_length(x) > n,
         paste0(str_sub(x, end = n - 1), "\u2026"),
         str_sub(x, end = n)) %>%
    str_pad(n)
}

mt <- mentions("cnn", n = 200, sinceID = cnn_ids[1])
length(mt)

df <- data_frame(mt = mt) %>%
  mutate(replyToSID = mt %>% map_chr2("replyToSID")) %>%
  filter(replyToSID == target_tweet_id)
df %>%
  mutate(id = mt %>% map_chr2("id"),
         screenName = mt %>% map_chr2("screenName"),
         text = mt %>% map_chr2("text")) %>%
  mutate(text = text %>% substr(13, 140) %>% trimws() %>% ellipsize(30)) %>%
  select(-replyToSID, -mt)

rtweet::lookup_tweets(cnn_ids[1])


##set name of tweeter to look at (this can be changed)
targettwittername <- "cnn"

##get this tweeter's timeline
tmls <- rtweet::get_timeline(targettwittername, n=100, retryonratelimit=TRUE)
##get their user id
targettwitteruserid <- as.numeric(select(rtweet::lookup_users(targettwittername), user_id))
##get ids of their tweets
tweetids <- select(tmls, status_id)
tweetids <- transform(tweetids, status_id_num=as.numeric(status_id))

##get list of followers (who are most likely to reply)
targetfollowers <- data.frame(rtweet::get_followers(targettwittername))

##clean up follower list to exclude those that have never tweeted and restricted access
user_lookup <- rtweet::lookup_users(targetfollowers)
users_with_tweets_and_unprotected <- filter(user_lookup, statuses_count != 0)
users_with_tweets_and_unprotected <- select(filter(users_with_tweets_and_unprotected, protected != "TRUE"), user_id)

targetfollowers <- filter(targetfollowers, user_id %in% users_with_tweets_and_unprotected$user_id)
options(scipen = 999)
getfollowersreplies <- function(x){
  follower <- as.numeric(x[1:500])
  followertl <- data.frame(rtweet::get_timeline(follower, n=3200, retryonratelimit=TRUE))
  followertl <- filter(followertl, reply_to_user_id == targettwitteruserid)
  followertl <- transform(followertl, reply_to_status_id_num=as.numeric(reply_to_user_id))
  join <- inner_join(followertl, tweetids, by=c("reply_to_status_id_num"="status_id_num"))
  replycounts <- data.frame(
                  join %>%
                    group_by(user_id, reply_to_status_id_num) %>%
                    summarise(n=n())
                  )
  return(replycounts)
}

tweet_replies <- do.call("rbind", lapply(targetfollowers$user_id, getfollowersreplies))

ss <- getfollowersreplies(x = targetfollowers$user_id)

flw <- rtweet::get_followers("realDonaldTrump", n = 2)

flw_tml <- vector("list", length(flw$user_id))
for (i in seq_along(flw$user_id)) {
  flw_tml[[i]] <- rtweet::get_timeline(
    flw$user_id[i], n = 3200
  )
  if (i %% 56 == 0L) {
    rl <- rate_limit("get_timeline")
    Sys.sleep(as.numeric(rl$reset, "secs"))
  }
  cat(i, " ")
}